/*
 * @(#)W3DomHypertextUnitBuilder.java	0.1 03/11/05
 *
 * Copyright 2004 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.html;

import java.io.*;
import java.util.*;

import javax.xml.parsers.*;
import org.w3c.dom.*;
import org.w3c.dom.traversal.*;
import org.xml.sax.*;
import org.w3c.tidy.*;

import org.tabularium.net.UrlNormalizer;

/**
 * An implementation of <code>HypertextUnitBuilder</code> based on the
 * <tt>Apache Xerces</tt> library.
 * <p>
 * This implementation does not tokenize text in Web pages or normalize links.
 * <p>
 * <tt>JTidy</tt> library is internally used before page parsing to clean up
 * malformed and faulty HTMLs.
 * <p>
 * See {@link http://xerces.apache.org/}
 * @author Fabio Gasparetti
 * @version 0.1 03/11/05
 */

public class W3DomHypertextUnitBuilder extends HypertextUnitBuilder {

	protected StringBuffer textBuf = null;

	protected String baseHref = null;

	protected HashSet htmlDelimSet = new HashSet();

	protected HashSet htmlUnitDelimSet = new HashSet();

	protected HashSet htmlUnitDblDelimSet = new HashSet();

	protected HypertextUnitFactory unitFactory = null;

	public W3DomHypertextUnitBuilder() {
		super();

	}

	/*
	 * (non-Javadoc)
	 * 
	 * @todo write exceptions
	 * 
	 * @see org.tabularium.text.html.HypertextUnitBuilder#build(java.lang.String,
	 *      java.lang.String)
	 */
	public HypertextUnit build(String url, String htmltext,
			HypertextUnitFactory unitFactory) throws Exception {
		this.unitFactory = unitFactory;
		HypertextUnit root = null;

		for (int i = 0; i < htmlDelim.length; i++) {
			this.htmlDelimSet.add(htmlDelim[i]);
		}
		for (int i = 0; i < htmlUnitDelim.length; i++) {
			this.htmlUnitDelimSet.add(htmlUnitDelim[i]);
		}
		for (int i = 0; i < htmlUnitDblDelim.length; i++) {
			this.htmlUnitDblDelimSet.add(htmlUnitDblDelim[i]);
		}

		try {
			DocumentBuilderFactory factory = DocumentBuilderFactory
					.newInstance();
			DocumentBuilder parser = factory.newDocumentBuilder();

			// Check for the traversal module
			DOMImplementation impl = parser.getDOMImplementation();
			if (!impl.hasFeature("traversal", "2.0")) {
				throw new Exception(
						"A DOM implementation that supports traversal is required.");
			}
			Tidy tidy = new Tidy();
			tidy.setXHTML(true);
			ByteArrayInputStream bis = new ByteArrayInputStream(htmltext
					.getBytes());
			ByteArrayOutputStream bos = new ByteArrayOutputStream();
			tidy.parse(bis, bos);
			bis.close();
			bis = new ByteArrayInputStream(bos.toByteArray());
			InputSource is = new InputSource(bis);
			is.setSystemId(url);
			Document doc = parser.parse(is);
			//bis.close();
			
			// Create the NodeIterator
			DocumentTraversal traversable = (DocumentTraversal) doc;
			TreeWalker tree = traversable.createTreeWalker(doc,
					NodeFilter.SHOW_ALL | NodeFilter.SHOW_ELEMENT
							| NodeFilter.SHOW_TEXT, null, true);

			textBuf = new StringBuffer();
			baseHref = url;
			root = unitFactory.newHypertextUnit();
			root.type = "ROOT";
			root.parent = null;
			traverseTree(tree.getRoot(), root);
			storeTextUnit(root);
			assignIds(root);

		} catch (SAXException e) {
			// System.out.println(url + " is not well-formed.");
			throw e;
		} catch (IOException e) {
			throw e;
			// System.out.println("Due to an IOException, the parser could not
			// check " + url);
		} catch (FactoryConfigurationError e) {
			throw e;
			// System.out.println("Could not locate a factory class");
		} catch (ParserConfigurationException e) {
			throw e;
			// System.out.println("Could not locate a JAXP parser");
		}
		return root;
	}

	protected void traverseTree(org.w3c.dom.Node node, HypertextUnit lastUnit) {

		if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
			if ("base".equalsIgnoreCase(node.getNodeName())) {
				storeTextUnit(lastUnit);
				org.w3c.dom.Node n1 = null;
				NamedNodeMap attrs = node.getAttributes();
				n1 = attrs.getNamedItem("href");
				if (n1 != null) {
					baseHref = n1.getNodeValue();
				}
			} else if ("a".equalsIgnoreCase(node.getNodeName())) {
				storeTextUnit(lastUnit);
				HypertextUnit unit = unitFactory.newHypertextUnit();
				unit.type = "LINK";
				unit.parent = lastUnit;
				org.w3c.dom.Node n1 = null;
				NamedNodeMap attrs = node.getAttributes();
				n1 = attrs.getNamedItem("title");
				if (n1 != null)
					unit.text = n1.getTextContent();
				n1 = attrs.getNamedItem("href");
				if (n1 != null) {
					String u = UrlNormalizer.unrelativizeUrl(baseHref, n1
							.getTextContent());
					unit.url = u;
				}
				// push
				lastUnit.content.add(unit);
				lastUnit = unit;
			} else if ("img".equalsIgnoreCase(node.getNodeName())) {
				storeTextUnit(lastUnit);
				// adds the alt-attribute text of the images
				HypertextUnit unit = unitFactory.newHypertextUnit();
				unit.type = "IMG";
				unit.parent = lastUnit;
				org.w3c.dom.Node n1 = null;
				NamedNodeMap attrs = node.getAttributes();
				n1 = attrs.getNamedItem("alt");
				if (n1 != null)
					unit.text = n1.getTextContent();
				n1 = attrs.getNamedItem("src");
				if (n1 != null) {
					String u = UrlNormalizer.unrelativizeUrl(baseHref, n1
							.getTextContent());
					unit.url = u;
				}
				lastUnit.content.add(unit);
			} else if ("title".equalsIgnoreCase(node.getNodeName())) {
				String text = node.getTextContent();
				// sometimes parser gets 2 title nodes
				// one with an empty text field
				if (text != null && text.length() > 0) {
					HypertextUnit unit = unitFactory.newHypertextUnit();
					unit.type = "TITLE";
					unit.text = text;
					unit.parent = lastUnit;
					lastUnit.content.add(unit);
				}
			}

			if (htmlDelimSet.contains(node.getNodeName().toUpperCase())) {
				storeTextUnit(lastUnit);
				HypertextUnit unit = unitFactory.newHypertextUnit();
				unit.type = node.getNodeName().toUpperCase();
				unit.parent = lastUnit;
				lastUnit.content.add(unit);
			} else if (htmlUnitDelimSet.contains(node.getNodeName()
					.toUpperCase())) {
				storeTextUnit(lastUnit);
				// add the delim
				HypertextUnit unit = unitFactory.newHypertextUnit();
				unit.type = node.getNodeName().toUpperCase();
				unit.parent = lastUnit;
				lastUnit.content.add(unit);
				lastUnit = unit;
			} else if (htmlUnitDblDelimSet.contains(node.getNodeName()
					.toUpperCase())) {
				storeTextUnit(lastUnit);
				// create a new unit
				HypertextUnit unit = unitFactory.newHypertextUnit();
				unit.type = node.getNodeName().toUpperCase();
				unit.parent = lastUnit;
				lastUnit.content.add(unit);
				lastUnit = unit;
			}
		} else if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
			textBuf.append(node.getTextContent() + " ");
		}
		NodeList childs = node.getChildNodes();
		for (int i = 0; i < childs.getLength(); i++) {
			traverseTree(childs.item(i), lastUnit);
		}
		// before going up one level store text
		storeTextUnit(lastUnit);
	}

	// collect last text and assign it to the last unit
	protected void storeTextUnit(HypertextUnit unit) {
		// collect last text and assign it to the last unit
		String text = textBuf.toString().trim();
		if (text.length() > 0) {
			HypertextUnit u = unitFactory.newHypertextUnit();
			u.text = text;
			u.type = "TEXT";
			u.parent = unit;
			// if we already inserted a text node, update it
			if (unit.content.size() > 0) {
				HypertextUnit past = (HypertextUnit) unit.content
						.get(unit.content.size() - 1);
				if (past.type.equalsIgnoreCase("TEXT")) {
					past.text = past.text.concat(" " + text);
				}
			} else {
				unit.content.add(u);
			}
			textBuf = new StringBuffer();
		}
	}

	// for debugging
	public static void main(String[] args) {
		try {
			InputStream is = new FileInputStream("thread.html");
			BufferedReader reader = new BufferedReader(
					new InputStreamReader(is));

			String line;
			StringBuffer doc = new StringBuffer();
			while ((line = reader.readLine()) != null) {
				doc.append(line + "\n");
			}
			is.close();
			W3DomHypertextUnitBuilder hu = new W3DomHypertextUnitBuilder();
			HypertextUnit root = hu
					.build("http://www.cnn.com/", doc.toString(), new DefaultHypertextUnitFactory());
			System.out.println("root:" + root.toIndentString());
			System.out.println();
			System.out.println();
			System.out.println(root.toIndentString());

		} catch (Exception ex) {
			System.out.println(ex.toString());
			ex.printStackTrace();
		}
	}
}
